# -*- coding: utf-8 -*-
'''
Created on Feb 18, 2013

@author: LONG HOANG GIANG
'''

from CrawlerLib2 import commonlib, html2text
from Model.storymodelv3 import StoryModel
from urlparse import urljoin
import datetime
import os
import re
import workerpool



URL = 'http://mylifengayhomqua.blogspot.com/{year}_{month}_01_archive.html'

def getChapter():
    years = [2012, 2013]
    months = [['04', '09', '10', '12'], ['01', '02']]
    data = []
    for i in range(len(years)):
        imonth = months[i]
        year = years[i]
        for month in imonth:
            url = URL.format(year=year, month=month)
            tree = commonlib.loadweb(url).build_tree()
            tmp = []
            for node in tree.xpath("//ul[@class='posts']/li/a"):
                title = commonlib.stringify(node)
                title = re.sub(r"Ngày hôm qua...đã từng *- *", "", title).strip('*')
                title = re.sub(r"Chap *", "CHƯƠNG ", title)
                title = commonlib.toUpper(title)
                href = node.get('href', '')
                if href == '': continue
                href = urljoin(url, href)
                tmp.append({'title': title, 'url': href})
            tmp.reverse()
            for item in tmp:
                data.append(item)
    for item in data:
        print item['title'], item['url']
    return data

def getDetail(item):
    url = item['url']
    tree = commonlib.loadweb(url).build_tree()
    detailNode = tree.xpath("//div[@class='post-body entry-content']")[0]
#    content = commonlib.stringify(detailNode)
    content = html2text.html2text(commonlib.Etree.tostring(detailNode)).encode('utf-8')
    content = '''<b>{0}</b><br /><hr /><br />{1}'''.format(item['title'], content)
    print content
    content = re.sub(r"\n", "<br />", content).strip()
    return content


def process():
    data = getChapter()
    
    def loadWeb(item):
        commonlib.loadweb(item['url'])
    pool = workerpool.WorkerPool(size=10)
    pool.map(loadWeb, data)
    pool.shutdown()
    pool.wait()
    
    a = StoryModel('/longhoanggiang/database/ngayhomquadatung')
    a.open(True)
    for chapter in data:
        detail = getDetail(chapter)
        if detail == '': raise Exception, "NULL CONTENT"
        a.add_story(chapter['title'], detail, 0)
    a.close()

if __name__ == '__main__':
    
#    getChapter()
#    getDetail({'url': 'http://mylifengayhomqua.blogspot.com/2013/02/ngay-hom-quaa-tung-my-daisy-chap-13.html'})
    process()
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)